import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.preprocessing import StandardScaler
import plotly.express as px
from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
from sklearn.decomposition import PCA
from sklearn.metrics import accuracy_score, classification_report, f1_score, ConfusionMatrixDisplay, silhouette_score

# Load the dataset
df = pd.read_csv("Spotify_Youtube.csv", index_col=0) 
# Display the first few rows of the dataset
print("First 5 rows of the dataset:")
df.head()

First 5 rows of the dataset:

df.info()

<class 'pandas.core.frame.DataFrame'>
Index: 20718 entries, 0 to 20717
Data columns (total 27 columns):
 #   Column            Non-Null Count  Dtype  
---  ------            --------------  -----  
 0   Artist            20718 non-null  object 
 1   Url_spotify       20718 non-null  object 
 2   Track             20718 non-null  object 
 3   Album             20718 non-null  object 
 4   Album_type        20718 non-null  object 
 5   Uri               20718 non-null  object 
 6   Danceability      20716 non-null  float64
 7   Energy            20716 non-null  float64
 8   Key               20716 non-null  float64
 9   Loudness          20716 non-null  float64
 10  Speechiness       20716 non-null  float64
 11  Acousticness      20716 non-null  float64
 12  Instrumentalness  20716 non-null  float64
 13  Liveness          20716 non-null  float64
 14  Valence           20716 non-null  float64
 15  Tempo             20716 non-null  float64
 16  Duration_ms       20716 non-null  float64
 17  Url_youtube       20248 non-null  object 
 18  Title             20248 non-null  object 
 19  Channel           20248 non-null  object 
 20  Views             20248 non-null  float64
 21  Likes             20177 non-null  float64
 22  Comments          20149 non-null  float64
 23  Description       19842 non-null  object 
 24  Licensed          20248 non-null  object 
 25  official_video    20248 non-null  object 
 26  Stream            20142 non-null  float64
dtypes: float64(15), object(12)
memory usage: 4.4+ MB

# Generate descriptive statistics
print("Descriptive Statistics:")
df.describe()

Descriptive Statistics:

# Check for missing values
print("Missing Values Count:")
df.isnull().sum()

Missing Values Count:

Artist                0
Url_spotify           0
Track                 0
Album                 0
Album_type            0
Uri                   0
Danceability          2
Energy                2
Key                   2
Loudness              2
Speechiness           2
Acousticness          2
Instrumentalness      2
Liveness              2
Valence               2
Tempo                 2
Duration_ms           2
Url_youtube         470
Title               470
Channel             470
Views               470
Likes               541
Comments            569
Description         876
Licensed            470
official_video      470
Stream              576
dtype: int64

df['Album_type'] = df['Album_type'].replace('compilation', 'album')

album_counts = df['Album_type'].value_counts().reset_index()
album_counts.columns = ['Album_type', 'Count']
album_counts['Percentage'] = album_counts['Count'] / album_counts['Count'].sum() * 100

fig = px.bar(album_counts, x='Album_type', y='Percentage', text='Percentage', color_discrete_sequence=['#00CC99'] * len(album_counts))
fig.update_traces(texttemplate='%{y:.2f}%', textposition='outside', hovertemplate='%{x}<br>Total Songs: %{customdata}', customdata=album_counts[['Count']], textfont_size=12)

# ⬇️ Reduce top margin & remove forced y-axis max
fig.update_layout(yaxis_title="Percentage from total (%)", margin=dict(t=50),  # reduced from 120
    title=dict(text='<b>Distribution Count of Album Types</b>', x=0.5, y=0.95, font=dict(family="Helvetica", size=25)),
    title_font_color='black',
    legend=dict(title_font_family="Helvetica", font=dict(size=15), orientation="h", yanchor="bottom", y=0.99, xanchor="right", x=0.65),
    uniformtext_minsize=10, uniformtext_mode='hide')

fig.show()

metrics = ['Views', 'Likes', 'Stream', 'Comments']
colors = ['lightgreen', 'salmon']  # album, single

fig, axes = plt.subplots(1, 4, figsize=(18, 5))

for i, metric in enumerate(metrics):
    values = df.groupby('Album_type')[metric].sum()
    axes[i].pie(values, labels=values.index, colors=colors, startangle=90,
                counterclock=False, wedgeprops={'width': 0.4, 'edgecolor': 'white'},
                autopct='%1.1f%%')
    axes[i].set_title(f'{metric} Distribution')

plt.tight_layout()
plt.show()

# Apply log1p (log(x + 1)) transformation to avoid log(0) issues
df_log = df.copy()
for col in ['Likes', 'Views', 'Comments', 'Stream']:
    df_log[col] = np.log1p(df_log[col])

# Create the pairplot
sns.pairplot(df_log[['Likes', 'Views', 'Comments', 'Stream', 'Album_type']],
             hue='Album_type',
             palette={'album': 'lightgreen', 'single': 'salmon'},
             corner=True)

plt.suptitle('Log-Transformed Pairwise Plot: Single vs Album', y=1.02)
plt.show()

sns.set(style='whitegrid')
fig, axes = plt.subplots(1, 2, figsize=(12, 5))

# Define custom color palette
custom_palette = {'album': 'lightgreen', 'single': 'salmon'}

# Scatter: Comments vs Likes
sns.scatterplot(data=df, x='Likes', y='Comments', hue='Album_type',
                ax=axes[0], alpha=0.9, s=20, edgecolors='black', legend='full', palette=custom_palette)
axes[0].set_title('Comments vs Likes')
axes[0].set_xscale('log'); axes[0].set_yscale('log')

# Scatter: Streams vs Views
sns.scatterplot(data=df, x='Views', y='Stream', hue='Album_type',
                ax=axes[1], alpha=0.9, s=20, edgecolors='black', legend='full', palette=custom_palette)
axes[1].set_title('Streams vs Views')
axes[1].set_xscale('log'); axes[1].set_yscale('log')

plt.tight_layout()
plt.show()

sns.heatmap(df[['Likes', 'Views', 'Comments', 'Stream']].corr(), annot=True, cmap='coolwarm')

<Axes: >

# Create the ratio column
df['Stream_to_Views'] = df['Stream'] / df['Views']

df_filtered = df[df['Stream_to_Views'] > 0]  # avoids division-by-zero log issues

plt.figure(figsize=(8, 6))
sns.violinplot(data=df_filtered, x='Album_type', y='Stream_to_Views',hue='Album_type',
               palette={'album': 'lightgreen', 'single': 'salmon'},
               inner='box', cut=0)
plt.yscale('log')
plt.title('Stream-to-Views Ratio by Album Type')
plt.xlabel('Album Type')
plt.ylabel('Stream / Views (log scale)')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

# Calculate Likes-to-Views ratio
df['Likes_to_Views'] = df['Likes'] / df['Views']

df_filtered = df[df['Likes_to_Views'].between(0, 1)]

plt.figure(figsize=(8, 6))
sns.violinplot(data=df_filtered, x='Album_type', y='Likes_to_Views',hue='Album_type', palette={'album': 'lightgreen', 'single': 'salmon'}, inner='box')

plt.title('Likes-to-Views Ratio by Album Type')
plt.xlabel('Album Type')
plt.ylabel('Likes / Views')
plt.yscale('log')

plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

# Create the ratio
df['Comments_to_Likes'] = df['Comments'] / df['Likes']

df_filtered = df[df['Comments_to_Likes'].between(0, 1)]

plt.figure(figsize=(8, 6))
sns.violinplot(data=df_filtered, x='Album_type', y='Comments_to_Likes',
               hue='Album_type', palette={'album': 'lightgreen', 'single': 'salmon'},
               inner='box', cut=0)
plt.yscale('log')
plt.title('Comments-to-Likes Ratio by Album Type')
plt.xlabel('Album Type')
plt.ylabel('Comments / Likes (log scale)')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

# Select the audio features and the label
audio_features = ['Danceability', 'Valence', 'Loudness', 'Energy', 'Tempo', 'Album_type']
df_audio = df[audio_features].dropna()

sns.pairplot(df_audio, hue='Album_type',
             palette={'album': 'lightgreen', 'single': 'salmon'},
             plot_kws={'alpha': 0.9, 's': 20})
plt.suptitle('Pairplot of Audio Features by Album Type', y=1.02)
plt.show()

plt.figure(figsize=(8, 6))
sns.violinplot(data=df, x='Album_type', y='Loudness',
               hue='Album_type', palette={'album': 'lightgreen', 'single': 'salmon'},
               inner='box', cut=0)
plt.title('Loudness Distribution by Album Type')
plt.ylabel('Loudness (dB)')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

# Create binary feature
df['Loudness_high'] = df['Loudness'] > df['Loudness'].median()

# Compute proportions
prop = df.groupby('Album_type')['Loudness_high'].mean().reset_index()

plt.figure(figsize=(6, 5))
sns.barplot(data=prop, x='Album_type', y='Loudness_high',hue='Album_type',
            palette={'album': 'lightgreen', 'single': 'salmon'})
plt.title('Proportion of Loud Songs by Album Type')
plt.ylabel('% Loud Songs')
plt.ylim(0, 1)
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

plt.figure(figsize=(8, 6))
sns.kdeplot(data=df, x='Danceability', y='Valence', hue='Album_type',
            fill=True, alpha=0.4, palette={'album': 'lightgreen', 'single': 'salmon'})
plt.title('Density of Danceability vs Valence by Album Type')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

plt.figure(figsize=(8, 6))
sns.kdeplot(data=df, x='Loudness', y='Energy',
            hue='Album_type', fill=True, alpha=0.4,
            palette={'album': 'lightgreen', 'single': 'salmon'})
plt.title('Density of Energy vs Loudness by Album Type')
plt.xlabel('Loudness (dB)')
plt.ylabel('Energy')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

popularity_df = df[(df[['Stream', 'Likes', 'Comments', 'Views']] > 0).all(axis=1)].copy()

popularity_df['Popularity'] = (
    popularity_df['Stream'].rank(pct=True) +
    popularity_df['Likes'].rank(pct=True) +
    popularity_df['Comments'].rank(pct=True) +
    popularity_df['Views'].rank(pct=True)
) / 4

# Replot the 2D density plot with no warning
plt.figure(figsize=(10, 6))
sns.kdeplot(
    data=popularity_df,
    x='Valence',
    y='Energy',
    weights=popularity_df['Popularity'],
    fill=True,
    cmap='viridis',
    thresh=0.01,
    levels=100
)
plt.title("Valence vs Energy (Weighted by Popularity)")
plt.xlabel("Valence")
plt.ylabel("Energy")
plt.tight_layout()
plt.show()

# Group by Album and aggregate total Streams and Views
album_stats = df.groupby('Album')[['Stream', 'Views']].sum().sort_values(by='Stream', ascending=False).head(15)

plt.figure(figsize=(12, 6))
album_stats.plot(kind='bar', figsize=(12, 6), color={'Stream': 'lightblue', 'Views': 'darkorange'})
plt.title("Top 15 Albums by Stream and Views")
plt.ylabel("Count")
plt.xlabel("Album")
plt.xticks(rotation=45, ha='right')
plt.grid(axis='y', linestyle='--', alpha=0.5)
plt.tight_layout()
plt.show()

<Figure size 1200x600 with 0 Axes>

licensed_counts = df.groupby(['Album_type', 'Licensed']).size().unstack()
licensed_counts.plot(kind='barh', stacked=True,
                     color=['lightcoral', 'lightblue'], figsize=(8, 5))
plt.title('Licensed Status Distribution by Album Type')
plt.xlabel('Number of Songs')
plt.tight_layout()
plt.show()

official_counts = df.groupby(['Album_type', 'official_video']).size().unstack()
official_counts.plot(kind='barh', stacked=True,
                     color=['lightcoral', 'lightblue'], figsize=(8, 5))
plt.title('Official Video Distribution by Album Type')
plt.xlabel('Number of Songs')
plt.tight_layout()
plt.show()

# Reload dataset to make sure we're working cleanly
df = pd.read_csv("Spotify_Youtube.csv")
df['Album_type'] = df['Album_type'].replace('compilation', 'album')

df['Likes'] = df['Likes'].fillna(0)
df['Comments'] = df['Comments'].fillna(0)

df.dropna(subset=['Views', 'Duration_ms', 'Loudness', 'Valence', 'Danceability', 'Energy','Stream','Title', 'Track', 'Description'], inplace=True)

df['Album_Song_Count'] = df.groupby('Album')['Track'].transform('count')

artist_view_avg = df.groupby('Artist')['Views'].transform('mean')
df['Avg_Artist_Song_Views'] = artist_view_avg

df['Song_Name_Length'] = df['Track'].astype(str).apply(lambda x: len(x.split()))

df['Total_Album_Length'] = df.groupby('Album')['Duration_ms'].transform('sum')

# Normalize loudness to [0,1] before averaging
loudness_norm = (df['Loudness'] - df['Loudness'].min()) / (df['Loudness'].max() - df['Loudness'].min())

# Compute Fitness_for_Clubs as the average of 4 features
df['Fitness_for_Clubs'] = pd.concat([
    df[['Danceability', 'Energy', 'Valence']],
    loudness_norm.to_frame('Loudness')
], axis=1).mean(axis=1)

# --- 8 Additional Recommended Features ---

df['Likes_to_Views'] = df['Likes'] / df['Views']

df['Stream_to_Views'] = df['Stream'] / df['Views']

df['Comments_to_Likes'] = df['Comments'] / df['Likes']

df['Loudness_High'] = df['Loudness'] > df['Loudness'].median()

df['Danceability_Valence'] = df['Danceability'] * df['Valence']

df['Popular_Site'] = (df['Views'] > df['Stream']).astype(int)


df['Is_Remix'] = df[['Track', 'Title', 'Description']].astype(str).apply(
    lambda row: 'remix' in ' '.join(row).lower(), axis=1)

df['Streams_per_Minute'] = df['Stream'] / (df['Duration_ms'] / 60000)

# Return updated dataframe shape and columns added
df.shape, df.columns[-13:].tolist()

((19298, 41),
 ['Album_Song_Count',
  'Avg_Artist_Song_Views',
  'Song_Name_Length',
  'Total_Album_Length',
  'Fitness_for_Clubs',
  'Likes_to_Views',
  'Stream_to_Views',
  'Comments_to_Likes',
  'Loudness_High',
  'Danceability_Valence',
  'Popular_Site',
  'Is_Remix',
  'Streams_per_Minute'])

df.info()

<class 'pandas.core.frame.DataFrame'>
Index: 19298 entries, 0 to 20717
Data columns (total 41 columns):
 #   Column                 Non-Null Count  Dtype  
---  ------                 --------------  -----  
 0   Unnamed: 0             19298 non-null  int64  
 1   Artist                 19298 non-null  object 
 2   Url_spotify            19298 non-null  object 
 3   Track                  19298 non-null  object 
 4   Album                  19298 non-null  object 
 5   Album_type             19298 non-null  object 
 6   Uri                    19298 non-null  object 
 7   Danceability           19298 non-null  float64
 8   Energy                 19298 non-null  float64
 9   Key                    19298 non-null  float64
 10  Loudness               19298 non-null  float64
 11  Speechiness            19298 non-null  float64
 12  Acousticness           19298 non-null  float64
 13  Instrumentalness       19298 non-null  float64
 14  Liveness               19298 non-null  float64
 15  Valence                19298 non-null  float64
 16  Tempo                  19298 non-null  float64
 17  Duration_ms            19298 non-null  float64
 18  Url_youtube            19298 non-null  object 
 19  Title                  19298 non-null  object 
 20  Channel                19298 non-null  object 
 21  Views                  19298 non-null  float64
 22  Likes                  19298 non-null  float64
 23  Comments               19298 non-null  float64
 24  Description            19298 non-null  object 
 25  Licensed               19298 non-null  object 
 26  official_video         19298 non-null  object 
 27  Stream                 19298 non-null  float64
 28  Album_Song_Count       19298 non-null  int64  
 29  Avg_Artist_Song_Views  19298 non-null  float64
 30  Song_Name_Length       19298 non-null  int64  
 31  Total_Album_Length     19298 non-null  float64
 32  Fitness_for_Clubs      19298 non-null  float64
 33  Likes_to_Views         19298 non-null  float64
 34  Stream_to_Views        19298 non-null  float64
 35  Comments_to_Likes      19272 non-null  float64
 36  Loudness_High          19298 non-null  bool   
 37  Danceability_Valence   19298 non-null  float64
 38  Popular_Site           19298 non-null  int32  
 39  Is_Remix               19298 non-null  bool   
 40  Streams_per_Minute     19298 non-null  float64
dtypes: bool(2), float64(23), int32(1), int64(3), object(12)
memory usage: 5.9+ MB

# Handle division by zero explicitly and safely
df['Likes_to_Views'] = np.where(df['Views'] > 0, df['Likes'] / df['Views'], 0)
df['Stream_to_Views'] = np.where(df['Views'] > 0, df['Stream'] / df['Views'], 0)
df['Comments_to_Likes'] = np.where(df['Likes'] > 0, df['Comments'] / df['Likes'], 0)

log_features = ['Views', 'Likes', 'Comments', 'Stream','Album_Song_Count', 'Avg_Artist_Song_Views',
    'Total_Album_Length', 'Streams_per_Minute','Stream_to_Views', 'Likes_to_Views','Comments_to_Likes','Duration_ms']
for col in log_features:
    df[f'Log_{col}'] = np.log1p(df[col])

df['Licensed'] = df['Licensed'].astype(str).map({'True': 1, 'False': 0})
df['official_video'] = df['official_video'].astype(str).map({'True': 1, 'False': 0})
df['Album_type_Label'] = df['Album_type'].map({'single': 1, 'album': 0})
df['Artist_freq'] = df['Artist'].map(df['Artist'].value_counts())
df['Channel_freq'] = df['Channel'].map(df['Channel'].value_counts())

# Drop remaining rows with NaNs
df.dropna(inplace=True)
df.reset_index(drop=True, inplace=True)
# Drop irrelevant columns after features are added
df.drop(['Description', 'Title', 'Url_youtube', 'Uri','Url_spotify','Track','Album_type','Unnamed: 0', 'Channel','Album','Artist'], axis=1, errors='ignore', inplace=True)

print('Data Cleaning Completed\n')
df.info()

Data Cleaning Completed

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 19298 entries, 0 to 19297
Data columns (total 45 columns):
 #   Column                     Non-Null Count  Dtype  
---  ------                     --------------  -----  
 0   Danceability               19298 non-null  float64
 1   Energy                     19298 non-null  float64
 2   Key                        19298 non-null  float64
 3   Loudness                   19298 non-null  float64
 4   Speechiness                19298 non-null  float64
 5   Acousticness               19298 non-null  float64
 6   Instrumentalness           19298 non-null  float64
 7   Liveness                   19298 non-null  float64
 8   Valence                    19298 non-null  float64
 9   Tempo                      19298 non-null  float64
 10  Duration_ms                19298 non-null  float64
 11  Views                      19298 non-null  float64
 12  Likes                      19298 non-null  float64
 13  Comments                   19298 non-null  float64
 14  Licensed                   19298 non-null  int64  
 15  official_video             19298 non-null  int64  
 16  Stream                     19298 non-null  float64
 17  Album_Song_Count           19298 non-null  int64  
 18  Avg_Artist_Song_Views      19298 non-null  float64
 19  Song_Name_Length           19298 non-null  int64  
 20  Total_Album_Length         19298 non-null  float64
 21  Fitness_for_Clubs          19298 non-null  float64
 22  Likes_to_Views             19298 non-null  float64
 23  Stream_to_Views            19298 non-null  float64
 24  Comments_to_Likes          19298 non-null  float64
 25  Loudness_High              19298 non-null  bool   
 26  Danceability_Valence       19298 non-null  float64
 27  Popular_Site               19298 non-null  int32  
 28  Is_Remix                   19298 non-null  bool   
 29  Streams_per_Minute         19298 non-null  float64
 30  Log_Views                  19298 non-null  float64
 31  Log_Likes                  19298 non-null  float64
 32  Log_Comments               19298 non-null  float64
 33  Log_Stream                 19298 non-null  float64
 34  Log_Album_Song_Count       19298 non-null  float64
 35  Log_Avg_Artist_Song_Views  19298 non-null  float64
 36  Log_Total_Album_Length     19298 non-null  float64
 37  Log_Streams_per_Minute     19298 non-null  float64
 38  Log_Stream_to_Views        19298 non-null  float64
 39  Log_Likes_to_Views         19298 non-null  float64
 40  Log_Comments_to_Likes      19298 non-null  float64
 41  Log_Duration_ms            19298 non-null  float64
 42  Album_type_Label           19298 non-null  int64  
 43  Artist_freq                19298 non-null  int64  
 44  Channel_freq               19298 non-null  int64  
dtypes: bool(2), float64(35), int32(1), int64(7)
memory usage: 6.3 MB

df.describe().T.sort_values('std', ascending=False)

# Features and target
X = df.drop(columns=['Album_type_Label'])
y = df['Album_type_Label']

# Remove any remaining infs/NaNs just in case
X.replace([np.inf, -np.inf], np.nan, inplace=True)
X.dropna(inplace=True)
y = y.loc[X.index]

# Split: 80/10/10
X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)
X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, stratify=y_temp, random_state=42)

rf_params = {'n_estimators': [100, 200],
    'max_depth': [None, 10, 20],
    'class_weight': ['balanced']}

rf_gs = GridSearchCV(RandomForestClassifier(random_state=42),rf_params,scoring='f1_macro', cv=3,n_jobs=-1)
rf_gs.fit(X_train, y_train)
print("Best RF Params:", rf_gs.best_params_)

Best RF Params: {'class_weight': 'balanced', 'max_depth': 20, 'n_estimators': 200}

rf_best = rf_gs.best_estimator_
y_val_pred_rf = rf_best.predict(X_val)
y_test_pred_rf = rf_best.predict(X_test)

ConfusionMatrixDisplay.from_estimator(rf_best, X_val, y_val, cmap='Blues')
plt.title("Random Forest - Validation Confusion Matrix")
plt.show()

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.utils.class_weight import compute_sample_weight
from sklearn.metrics import f1_score

# Step 1: Define parameter grid
gboost_params = {
    'n_estimators': [100, 200],
    'learning_rate': [0.05, 0.1],
    'max_depth': [3, 5]
}

# Step 2: GridSearchCV using macro F1 to tune for imbalance-aware evaluation
gboost_gs = GridSearchCV(
    GradientBoostingClassifier(random_state=42),
    gboost_params,
    scoring='f1_macro',
    cv=3,
    n_jobs=-1
)
gboost_gs.fit(X_train, y_train)

# Step 3: Get best hyperparameters from grid search
best_params = gboost_gs.best_params_

# Step 4: Compute sample weights for class imbalance
sample_weights = compute_sample_weight(class_weight='balanced', y=y_train)

# Step 5: Re-train the model with sample weights using best hyperparameters
gboost_best = GradientBoostingClassifier(**best_params, random_state=42)
gboost_best.fit(X_train, y_train, sample_weight=sample_weights)

# Step 6: Make predictions
y_val_pred_gb = gboost_best.predict(X_val)
y_test_pred_gb = gboost_best.predict(X_test)

# Step 7: (Optional) Evaluate F1 scores
val_f1 = f1_score(y_val, y_val_pred_gb, average='macro')
test_f1 = f1_score(y_test, y_test_pred_gb, average='macro')

print("Validation Macro F1 Score:", val_f1)
print("Test Macro F1 Score:", test_f1)

Validation Macro F1 Score: 0.8189149393735442
Test Macro F1 Score: 0.793246115172154

# Confusion Matrix - Gradient Boosting
ConfusionMatrixDisplay.from_estimator(gboost_best, X_val, y_val, cmap='Purples')
plt.title("GBoost - Validation Confusion Matrix")
plt.show()

# Scale features for SVM
scaler = StandardScaler()
X_train_svm = scaler.fit_transform(X_train)
X_val_svm = scaler.transform(X_val)
X_test_svm = scaler.transform(X_test)

svm_params = {'C': [0.1, 1, 10, 50],'gamma': [0.01, 0.1, 1, 'scale', 'auto'],'kernel': ['rbf'],'class_weight': ['balanced']}
svm_gs = GridSearchCV(SVC(), svm_params, scoring='f1_macro', cv=3, n_jobs=-1)
svm_gs.fit(X_train_svm, y_train)
print("Best SVM Params:", svm_gs.best_params_)

Best SVM Params: {'C': 1, 'class_weight': 'balanced', 'gamma': 0.1, 'kernel': 'rbf'}

svm_best = svm_gs.best_estimator_
y_val_pred_svm = svm_best.predict(X_val_svm)
y_test_pred_svm = svm_best.predict(X_test_svm)

# Confusion Matrix - SVM
ConfusionMatrixDisplay.from_estimator(svm_best, X_val_svm, y_val, cmap='Greens')
plt.title("SVM - Validation Confusion Matrix")
plt.show()

# Refit SVM on raw (unscaled) data for compatibility with VotingClassifier
svm_for_ensemble = SVC(C=1, gamma='scale', kernel='rbf', class_weight='balanced', probability=True, random_state=42)
svm_for_ensemble.fit(X_train, y_train)

# Define voting classifier
voting = VotingClassifier(
    estimators=[('rf', rf_best),('gb', gboost_best),('svm', svm_for_ensemble)],voting='hard')

# Fit ensemble
voting.fit(X_train, y_train)

# Validation predictions
y_val_pred_vote = voting.predict(X_val)
print("VotingClassifier - Validation Accuracy:", accuracy_score(y_val, y_val_pred_vote))
print(classification_report(y_val, y_val_pred_vote))

# Test predictions
y_test_pred_vote = voting.predict(X_test)
print("VotingClassifier - Test Accuracy:", accuracy_score(y_test, y_test_pred_vote))
print(classification_report(y_test, y_test_pred_vote))

VotingClassifier - Validation Accuracy: 0.8725388601036269
              precision    recall  f1-score   support

           0       0.90      0.93      0.92      1464
           1       0.76      0.68      0.72       466

    accuracy                           0.87      1930
   macro avg       0.83      0.81      0.82      1930
weighted avg       0.87      0.87      0.87      1930

VotingClassifier - Test Accuracy: 0.8601036269430051
              precision    recall  f1-score   support

           0       0.90      0.92      0.91      1465
           1       0.73      0.67      0.70       465

    accuracy                           0.86      1930
   macro avg       0.81      0.79      0.80      1930
weighted avg       0.86      0.86      0.86      1930

# Generate macro F1
metrics_summary = {
    'Model': ['Random Forest', 'GBoost', 'SVM'],
    'Val Accuracy': [
        accuracy_score(y_val, y_val_pred_rf),
        accuracy_score(y_val, y_val_pred_gb),
        accuracy_score(y_val, y_val_pred_svm)
    ],
    'Val Macro F1': [
        f1_score(y_val, y_val_pred_rf, average='macro'),
        f1_score(y_val, y_val_pred_gb, average='macro'),
        f1_score(y_val, y_val_pred_svm, average='macro')
    ],
    'Test Accuracy': [
        accuracy_score(y_test, y_test_pred_rf),
        accuracy_score(y_test, y_test_pred_gb),
        accuracy_score(y_test, y_test_pred_svm)
    ],
    'Test Macro F1': [
        f1_score(y_test, y_test_pred_rf, average='macro'),
        f1_score(y_test, y_test_pred_gb, average='macro'),
        f1_score(y_test, y_test_pred_svm, average='macro')
    ]
}
# Add VotingClassifier to the summary
metrics_summary['Model'].append('Voting Ensemble')
metrics_summary['Val Accuracy'].append(accuracy_score(y_val, y_val_pred_vote))
metrics_summary['Val Macro F1'].append(f1_score(y_val, y_val_pred_vote, average='macro'))
metrics_summary['Test Accuracy'].append(accuracy_score(y_test, y_test_pred_vote))
metrics_summary['Test Macro F1'].append(f1_score(y_test, y_test_pred_vote, average='macro'))

summary_df = pd.DataFrame(metrics_summary)

# Validation Plot
summary_df = pd.DataFrame(metrics_summary)
summary_df.set_index('Model')[['Val Accuracy', 'Val Macro F1']].plot(kind='bar', figsize=(8, 5), color=['steelblue', 'seagreen'])
plt.title('Model Comparison: Accuracy and Macro F1')
plt.ylabel('Score')
plt.ylim(0.75, 0.90)
plt.xticks(rotation=0)
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.tight_layout()
plt.show()

# Test Plot
summary_df.set_index('Model')[['Test Accuracy', 'Test Macro F1']].plot(kind='bar', figsize=(8, 5), color=['orange', 'tomato'])
plt.title('Model Comparison: Test Accuracy and Macro F1')
plt.ylabel('Score')
plt.ylim(0.75, 0.90)
plt.xticks(rotation=0)
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.tight_layout()
plt.show()

importance_df = pd.DataFrame({
    'Feature': X_train.columns,
    'Importance': rf_best.feature_importances_
}).sort_values(by='Importance', ascending=False)

plt.figure(figsize=(10, 10))
plt.barh(importance_df['Feature'], importance_df['Importance'])
plt.gca().invert_yaxis()
plt.title('Feature Importance (Random Forest)')
plt.xlabel('Importance Score')
plt.tight_layout()
plt.show()

correlations = df.corr(numeric_only=True)['Album_type_Label'].sort_values(ascending=False)
print(correlations)

Album_type_Label             1.000000
Log_Likes_to_Views           0.214699
Likes_to_Views               0.211173
Is_Remix                     0.185334
Danceability                 0.157187
Loudness                     0.135283
Loudness_High                0.112715
Fitness_for_Clubs            0.107264
Channel_freq                 0.093319
Popular_Site                 0.086422
Energy                       0.085514
official_video               0.080704
Log_Avg_Artist_Song_Views    0.059551
Danceability_Valence         0.058708
Speechiness                  0.052014
Artist_freq                  0.042403
Log_Likes                    0.034574
Key                          0.030555
Song_Name_Length             0.029157
Avg_Artist_Song_Views        0.020909
Likes                        0.013131
Log_Comments                 0.011887
Tempo                        0.009077
Valence                      0.003521
Comments                    -0.002607
Stream_to_Views             -0.004636
Licensed                    -0.006692
Comments_to_Likes           -0.008392
Liveness                    -0.013966
Log_Comments_to_Likes       -0.020703
Views                       -0.024286
Log_Views                   -0.037140
Instrumentalness            -0.037456
Acousticness                -0.052742
Log_Stream_to_Views         -0.058899
Streams_per_Minute          -0.062940
Duration_ms                 -0.070484
Stream                      -0.080229
Log_Duration_ms             -0.119482
Total_Album_Length          -0.133735
Log_Streams_per_Minute      -0.135013
Log_Stream                  -0.157638
Album_Song_Count            -0.203960
Log_Album_Song_Count        -0.298027
Log_Total_Album_Length      -0.339924
Name: Album_type_Label, dtype: float64

#Feature Correlation Matrix
plt.figure(figsize=(12,10))
sns.heatmap(df.corr(numeric_only=True), cmap='coolwarm', center=0)
plt.title("Feature Correlation Matrix")
plt.tight_layout()
plt.show()

features_to_drop = ['Licensed', 'Channel', 'Key', 'Song_Name_Length', 'Liveness', 'Popular_Site']
X_reduced = X.drop(columns=features_to_drop, errors='ignore')

X_train_r, X_temp_r, y_train_r, y_temp_r = train_test_split(X_reduced, y, test_size=0.2, random_state=42, stratify=y)
X_val_r, X_test_r, y_val_r, y_test_r = train_test_split(X_temp_r, y_temp_r, test_size=0.5, random_state=42, stratify=y_temp_r)

rf_reduced = RandomForestClassifier(random_state=42)
rf_reduced.fit(X_train_r, y_train_r)
y_val_pred_r = rf_reduced.predict(X_val_r)

print("Validation Accuracy (Reduced Features):", accuracy_score(y_val_r, y_val_pred_r))
print(classification_report(y_val_r, y_val_pred_r))

Validation Accuracy (Reduced Features): 0.8797927461139896
              precision    recall  f1-score   support

           0       0.90      0.95      0.92      1464
           1       0.80      0.67      0.73       466

    accuracy                           0.88      1930
   macro avg       0.85      0.81      0.83      1930
weighted avg       0.88      0.88      0.88      1930

# Define focused features for clustering
features_to_use = [
    'Danceability', 'Valence', 'Energy', 'Loudness', 'Tempo',
    'Speechiness', 'Acousticness', 'Instrumentalness', 'Liveness',
    'Fitness_for_Clubs', 'Danceability_Valence',
    'Log_Likes_to_Views', 'Log_Stream_to_Views', 'Log_Comments_to_Likes',
    'Log_Views', 'Log_Stream', 'Log_Avg_Artist_Song_Views',
    'Log_Total_Album_Length', 'Log_Duration_ms', 'Log_Streams_per_Minute'
]

X_cluster = df[features_to_use]

# Standardize features
scaler = StandardScaler()
X_cluster_scaled = scaler.fit_transform(X_cluster)

# Try K values
silhouette_scores = []
inertias = []
K_range = range(2, 11)

for k in K_range:
    kmeans = KMeans(n_clusters=k, random_state=42, n_init='auto')
    kmeans.fit(X_cluster_scaled)
    inertias.append(kmeans.inertia_)
    silhouette_scores.append(silhouette_score(X_cluster_scaled, kmeans.labels_))

# Plot results
fig, ax = plt.subplots(1, 2, figsize=(12, 5))

ax[0].plot(K_range, inertias, marker='o')
ax[0].set_title('Elbow Method (Inertia)')
ax[0].set_xlabel('Number of clusters')
ax[0].set_ylabel('Inertia')

ax[1].plot(K_range, silhouette_scores, marker='o', color='green')
ax[1].set_title('Silhouette Score')
ax[1].set_xlabel('Number of clusters')
ax[1].set_ylabel('Silhouette Score')

plt.tight_layout()
plt.show()

# Try DBSCAN with a chosen eps value (tune manually)
dbscan = DBSCAN(eps=2.0, min_samples=5)
dbscan_labels = dbscan.fit_predict(X_cluster_scaled)

# Filter noise for silhouette score (label = -1 is noise)
valid = dbscan_labels != -1
if valid.sum() > 1:
    print("DBSCAN Silhouette Score:", silhouette_score(X_cluster_scaled[valid], dbscan_labels[valid]))
else:
    print("DBSCAN produced too few valid clusters.")

# Fit K-Means with optimal K (replace K=3 with your best choice)
kmeans_final = KMeans(n_clusters=3, random_state=42, n_init='auto')
cluster_labels = kmeans_final.fit_predict(X_cluster_scaled)

DBSCAN Silhouette Score: -0.1688632751033337

# Reduce dimensions
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_cluster_scaled)

# Plot PCA
plt.figure(figsize=(8, 6))
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=cluster_labels, cmap='Set2', alpha=0.7)
plt.title('PCA Projection of K-Means Clusters')
plt.xlabel('PC 1')
plt.ylabel('PC 2')
plt.grid(True)
plt.tight_layout()
plt.show()

# Add cluster labels to original data
df['Cluster'] = cluster_labels

# Group by cluster and compare feature means
cluster_summary = df.groupby('Cluster')[features_to_use].mean().round(2)
print(cluster_summary)

# Heatmap of feature means per cluster
plt.figure(figsize=(12, 6))
sns.heatmap(cluster_summary.T, annot=True, fmt=".2f", cmap="coolwarm")
plt.title("Feature Means by Cluster")
plt.tight_layout()
plt.show()

         Danceability  Valence  Energy  Loudness   Tempo  Speechiness  \
Cluster                                                                 
0                0.67     0.62    0.70     -6.77  121.97         0.12   
1                0.64     0.55    0.69     -6.24  122.21         0.09   
2                0.43     0.25    0.30    -14.69  111.39         0.05   

         Acousticness  Instrumentalness  Liveness  Fitness_for_Clubs  \
Cluster                                                                
0                0.22              0.03      0.21               0.71   
1                0.22              0.01      0.19               0.68   
2                0.70              0.26      0.16               0.41   

         Danceability_Valence  Log_Likes_to_Views  Log_Stream_to_Views  \
Cluster                                                                  
0                        0.42                0.02                 2.57   
1                        0.36                0.01                 1.24   
2                        0.12                0.01                 3.27   

         Log_Comments_to_Likes  Log_Views  Log_Stream  \
Cluster                                                 
0                         0.04      14.10       16.34   
1                         0.03      17.95       18.59   
2                         0.03      14.23       17.26   

         Log_Avg_Artist_Song_Views  Log_Total_Album_Length  Log_Duration_ms  \
Cluster                                                                       
0                            16.65                   12.93            12.21   
1                            18.20                   13.10            12.32   
2                            16.19                   13.03            12.23   

         Log_Streams_per_Minute  
Cluster                          
0                         15.13  
1                         17.27  
2                         16.04

# Calculate variation of each feature across clusters
feature_spreads = cluster_summary.T.std(axis=1).sort_values(ascending=False)
print(feature_spreads.head(5))  # Top 5 most varying features

Tempo                     6.178813
Loudness                  4.733036
Log_Views                 2.186237
Log_Stream                1.131209
Log_Streams_per_Minute    1.073980
dtype: float64

plt.figure(figsize=(8, 6))
sns.scatterplot(
    data=df,
    x='Danceability',
    y='Energy',
    hue='Cluster',
    palette='Set2',
    alpha=0.7
)
plt.title("Danceability vs. Energy by Cluster")
plt.xlabel("Danceability")
plt.ylabel("Energy")
plt.grid(True)
plt.tight_layout()
plt.show()

# Run Agglomerative clustering
agg = AgglomerativeClustering(n_clusters=3)
agg_labels = agg.fit_predict(X_cluster_scaled)

# Evaluate
silhouette_agg = silhouette_score(X_cluster_scaled, agg_labels)
print("Silhouette Score (Agglomerative):", silhouette_agg)

Silhouette Score (Agglomerative): 0.04545357256431154

# cluster_labels is from: cluster_labels = kmeans_final.fit_predict(X_cluster_scaled)
silhouette_kmeans = silhouette_score(X_cluster_scaled, cluster_labels)
print("Silhouette Score (K-Means):", silhouette_kmeans)

Silhouette Score (K-Means): 0.12967693891098206

# Load original data (only once!)
df_raw = pd.read_csv("Spotify_Youtube.csv") 

# Extract Artist column only
track_artist = df_raw[['Artist']].copy()
track_artist.index = df_raw.index  

# Copy cleaned df
df_partE = df.copy()

# Attach Artist to cleaned df (only for Part E)
df_partE['Artist'] = track_artist

# Drop missing artists (just in case)
df_partE = df_partE.dropna(subset=['Artist'])

# Group everything at once into a fresh artist-level DataFrame
artist_df = df_partE.groupby('Artist').agg(
    Avg_Views=('Views', 'mean'),
    Avg_Streams=('Stream', 'mean'),
    Avg_Likes=('Likes', 'mean'),
    Avg_Comments=('Comments', 'mean'),
    Avg_Fitness=('Fitness_for_Clubs', 'mean'),
    Avg_Danceability=('Danceability', 'mean'),
    Avg_Energy=('Energy', 'mean'),
    Avg_Loudness=('Loudness', 'mean'),
    Avg_Tempo=('Tempo', 'mean'),
    Avg_Log_StreamToViews=('Log_Stream_to_Views', 'mean'),
    Avg_Log_LikesToViews=('Log_Likes_to_Views', 'mean'),
    Avg_Log_CommentsToLikes=('Log_Comments_to_Likes', 'mean'),
    Avg_DanceValence=('Danceability_Valence', 'mean'),
    Loudness_High_Rate=('Loudness_High', 'mean'),
    Total_Songs=('Album_type_Label', 'count'),
    
).reset_index()

# Correct binary label for Part E: High Engagement
engagement_median = artist_df['Avg_Log_LikesToViews'].median()
artist_df['High_Engagement'] = (artist_df['Avg_Log_LikesToViews'] > engagement_median).astype(int)

# Visualizing the number of songs per artist
artist_counts = df_partE['Artist'].value_counts()

plt.figure(figsize=(10, 5))
sns.histplot(artist_counts, bins=30, kde=False)
plt.title("Distribution of Number of Songs per Artist")
plt.xlabel("Number of Songs")
plt.ylabel("Number of Artists")
plt.grid(True)
plt.tight_layout()
plt.show()

# Distribution of log(likes/views) per artist
plt.figure(figsize=(10, 5))
sns.histplot(artist_df['Avg_Log_LikesToViews'], bins=30, kde=True)
plt.axvline(artist_df['Avg_Log_LikesToViews'].median(), color='red', linestyle='--', label='Median')
plt.title("Distribution of Average Log Likes-to-Views Ratio per Artist")
plt.xlabel("Avg Log(Likes / Views)")
plt.ylabel("Number of Artists")
plt.legend()
plt.tight_layout()
plt.show()

# Visualizing high vs low engagement label distribution
sns.countplot(x='High_Engagement', data=artist_df,hue='High_Engagement', palette='Set2')
plt.title("Distribution of High Engagement Labels")
plt.xlabel("High Engagement (1 = High, 0 = Low)")
plt.ylabel("Number of Artists")
plt.xticks([0, 1], ['Low Engagement', 'High Engagement'])
plt.tight_layout()
plt.show()

# Step 1: Define features and label
# All features except label, original source of label, and non-numerics
excluded = ['High_Engagement', 'Avg_Log_LikesToViews', 'Artist','Total_Songs']

all_features = [col for col in artist_df.columns 
                if col not in excluded and artist_df[col].dtype in ['float64', 'int64']]

X = artist_df[all_features]
y = artist_df['High_Engagement']


# Step 2: Split into train/val/test (80/10/10)
X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.10, stratify=y, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1111, stratify=y_trainval, random_state=42)

# Step 3: Grid search with balanced RF
param_grid = {
    'n_estimators': [100, 200],
    'max_depth': [5, 10, None],
    'min_samples_split': [2, 5],
    'min_samples_leaf': [1, 2],
    'class_weight': ['balanced']
}

grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    scoring='f1_macro',
    cv=StratifiedKFold(n_splits=3, shuffle=True, random_state=42),
    n_jobs=-1,
    verbose=1
)

# Step 4: Fit model and evaluate
grid_search.fit(X_train, y_train)
best_model = grid_search.best_estimator_

y_pred = best_model.predict(X_test)

# Step 5: Evaluation
print("Best Params:", grid_search.best_params_)
print("\nClassification Report:\n", classification_report(y_test, y_pred))

Fitting 3 folds for each of 24 candidates, totalling 72 fits
Best Params: {'class_weight': 'balanced', 'max_depth': None, 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 200}

Classification Report:
               precision    recall  f1-score   support

           0       0.77      0.77      0.77        97
           1       0.77      0.77      0.77        97

    accuracy                           0.77       194
   macro avg       0.77      0.77      0.77       194
weighted avg       0.77      0.77      0.77       194

# Predict on training data for heatmap comparison
y_train_pred = best_model.predict(X_train)

# Compute confusion matrices
cm_test = confusion_matrix(y_test, y_pred)
cm_train = confusion_matrix(y_train, y_train_pred)

# Plot heatmaps
fig, axes = plt.subplots(1, 2, figsize=(14, 6))

# Test set
sns.heatmap(cm_test, annot=True, fmt='d', cmap='Blues', ax=axes[0])
axes[0].set_title('Test Set Confusion Matrix')
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')
axes[0].set_xticklabels(['Low Engagement', 'High Engagement'])
axes[0].set_yticklabels(['Low Engagement', 'High Engagement'])

# Train set
sns.heatmap(cm_train, annot=True, fmt='d', cmap='Greens', ax=axes[1])
axes[1].set_title('Train Set Confusion Matrix')
axes[1].set_xlabel('Predicted')
axes[1].set_ylabel('Actual')
axes[1].set_xticklabels(['Low Engagement', 'High Engagement'])
axes[1].set_yticklabels(['Low Engagement', 'High Engagement'])

plt.tight_layout()
plt.show()

# Step 6: Feature Importance
importances = best_model.feature_importances_
feat_imp = pd.Series(importances, index=X.columns).sort_values(ascending=False)


plt.figure(figsize=(10,6))
sns.barplot(x=feat_imp, y=feat_imp.index)
plt.title("Feature Importance (Random Forest - High Engagement)")
plt.xlabel("Importance Score")
plt.tight_layout()
plt.show()

gboost = GradientBoostingClassifier(
    n_estimators=200,
    max_depth=5,
    learning_rate=0.1
)
gboost.fit(X_train, y_train)
y_pred = gboost.predict(X_test)

# Evaluate
print(classification_report(y_test, y_pred))

              precision    recall  f1-score   support

           0       0.79      0.76      0.77        97
           1       0.77      0.79      0.78        97

    accuracy                           0.78       194
   macro avg       0.78      0.78      0.78       194
weighted avg       0.78      0.78      0.78       194

param_grid = {
    'n_estimators': [100, 200, 300],
    'learning_rate': [0.05, 0.1, 0.2],
    'max_depth': [3, 5, 7],
    'min_samples_split': [2, 5],
    'min_samples_leaf': [1, 2]
}

gboost = GradientBoostingClassifier(random_state=42)

grid = GridSearchCV(
    estimator=gboost,
    param_grid=param_grid,
    scoring='f1_macro',
    cv=3,
    n_jobs=-1,
    verbose=1
)

grid.fit(X_train, y_train)

# Evaluate best model
best_gboost = grid.best_estimator_
y_pred = best_gboost.predict(X_test)

from sklearn.metrics import classification_report, confusion_matrix
print("Best Params:", grid.best_params_)
print("\nClassification Report:\n", classification_report(y_test, y_pred))

Fitting 3 folds for each of 108 candidates, totalling 324 fits
Best Params: {'learning_rate': 0.05, 'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 200}

Classification Report:
               precision    recall  f1-score   support

           0       0.80      0.79      0.80        97
           1       0.80      0.80      0.80        97

    accuracy                           0.80       194
   macro avg       0.80      0.80      0.80       194
weighted avg       0.80      0.80      0.80       194

# Predictions
y_test_pred = best_gboost.predict(X_test)
y_val_pred = best_gboost.predict(X_val)

# Confusion matrices
cm_test = confusion_matrix(y_test, y_test_pred)
cm_val = confusion_matrix(y_val, y_val_pred)

# Plot both
fig, axes = plt.subplots(1, 2, figsize=(12, 5))

# Test set
sns.heatmap(cm_test, annot=True, fmt='d', cmap='Blues', ax=axes[0])
axes[0].set_title("Test Set Confusion Matrix")
axes[0].set_xlabel("Predicted")
axes[0].set_ylabel("Actual")
axes[0].set_xticklabels(['Low Engagement', 'High Engagement'])
axes[0].set_yticklabels(['Low Engagement', 'High Engagement'])

# Validation set
sns.heatmap(cm_val, annot=True, fmt='d', cmap='Greens', ax=axes[1])
axes[1].set_title("Validation Set Confusion Matrix")
axes[1].set_xlabel("Predicted")
axes[1].set_ylabel("Actual")
axes[1].set_xticklabels(['Low Engagement', 'High Engagement'])
axes[1].set_yticklabels(['Low Engagement', 'High Engagement'])

plt.tight_layout()
plt.show()

	Artist	Url_spotify	Track	Album	Album_type	Uri	Danceability	Energy	Key	Loudness	...	Url_youtube	Title	Channel	Views	Likes	Comments	Description	Licensed	official_video	Stream
0	Gorillaz	https://open.spotify.com/artist/3AA28KZvwAUcZu...	Feel Good Inc.	Demon Days	album	spotify:track:0d28khcov6AiegSCpG5TuT	0.818	0.705	6.0	-6.679	...	https://www.youtube.com/watch?v=HyHNuVaZJ-k	Gorillaz - Feel Good Inc. (Official Video)	Gorillaz	693555221.0	6220896.0	169907.0	Official HD Video for Gorillaz' fantastic trac...	True	True	1.040235e+09
1	Gorillaz	https://open.spotify.com/artist/3AA28KZvwAUcZu...	Rhinestone Eyes	Plastic Beach	album	spotify:track:1foMv2HQwfQ2vntFf9HFeG	0.676	0.703	8.0	-5.815	...	https://www.youtube.com/watch?v=yYDmaexVHic	Gorillaz - Rhinestone Eyes [Storyboard Film] (...	Gorillaz	72011645.0	1079128.0	31003.0	The official video for Gorillaz - Rhinestone E...	True	True	3.100837e+08
2	Gorillaz	https://open.spotify.com/artist/3AA28KZvwAUcZu...	New Gold (feat. Tame Impala and Bootie Brown)	New Gold (feat. Tame Impala and Bootie Brown)	single	spotify:track:64dLd6rVqDLtkXFYrEUHIU	0.695	0.923	1.0	-3.930	...	https://www.youtube.com/watch?v=qJa-VFwPpYA	Gorillaz - New Gold ft. Tame Impala & Bootie B...	Gorillaz	8435055.0	282142.0	7399.0	Gorillaz - New Gold ft. Tame Impala & Bootie B...	True	True	6.306347e+07
3	Gorillaz	https://open.spotify.com/artist/3AA28KZvwAUcZu...	On Melancholy Hill	Plastic Beach	album	spotify:track:0q6LuUqGLUiCPP1cbdwFs3	0.689	0.739	2.0	-5.810	...	https://www.youtube.com/watch?v=04mfKJWDSzI	Gorillaz - On Melancholy Hill (Official Video)	Gorillaz	211754952.0	1788577.0	55229.0	Follow Gorillaz online:\nhttp://gorillaz.com \...	True	True	4.346636e+08
4	Gorillaz	https://open.spotify.com/artist/3AA28KZvwAUcZu...	Clint Eastwood	Gorillaz	album	spotify:track:7yMiX7n9SBvadzox8T5jzT	0.663	0.694	10.0	-8.627	...	https://www.youtube.com/watch?v=1V_xRb0x9aw	Gorillaz - Clint Eastwood (Official Video)	Gorillaz	618480958.0	6197318.0	155930.0	The official music video for Gorillaz - Clint ...	True	True	6.172597e+08

	Danceability	Energy	Key	Loudness	Speechiness	Acousticness	Instrumentalness	Liveness	Valence	Tempo	Duration_ms	Views	Likes	Comments	Stream
count	20716.000000	20716.000000	20716.000000	20716.000000	20716.000000	20716.000000	20716.000000	20716.000000	20716.000000	20716.000000	2.071600e+04	2.024800e+04	2.017700e+04	2.014900e+04	2.014200e+04
mean	0.619777	0.635250	5.300348	-7.671680	0.096456	0.291535	0.055962	0.193521	0.529853	120.638340	2.247176e+05	9.393782e+07	6.633411e+05	2.751899e+04	1.359422e+08
std	0.165272	0.214147	3.576449	4.632749	0.111960	0.286299	0.193262	0.168531	0.245441	29.579018	1.247905e+05	2.746443e+08	1.789324e+06	1.932347e+05	2.441321e+08
min	0.000000	0.000020	0.000000	-46.251000	0.000000	0.000001	0.000000	0.014500	0.000000	0.000000	3.098500e+04	0.000000e+00	0.000000e+00	0.000000e+00	6.574000e+03
25%	0.518000	0.507000	2.000000	-8.858000	0.035700	0.045200	0.000000	0.094100	0.339000	97.002000	1.800095e+05	1.826002e+06	2.158100e+04	5.090000e+02	1.767486e+07
50%	0.637000	0.666000	5.000000	-6.536000	0.050500	0.193000	0.000002	0.125000	0.537000	119.965000	2.132845e+05	1.450110e+07	1.244810e+05	3.277000e+03	4.968298e+07
75%	0.740250	0.798000	8.000000	-4.931000	0.103000	0.477250	0.000463	0.237000	0.726250	139.935000	2.524430e+05	7.039975e+07	5.221480e+05	1.436000e+04	1.383581e+08
max	0.975000	1.000000	11.000000	0.920000	0.964000	0.996000	1.000000	1.000000	0.993000	243.372000	4.676058e+06	8.079649e+09	5.078865e+07	1.608314e+07	3.386520e+09

Feature Name	Formula / Description	Reason to Add
`Album_Song_Count`	Number of songs in the current song’s album	Albums typically have more than one track; singles only have one
`Avg_Artist_Song_Views`	Average views of all songs by the current artist	Reflects artist popularity, which may impact release format
`Song_Name_Length`	Number of words in the track name	Singles might have shorter, catchier names
`Total_Album_Length`	Total duration (sum of durations) of all songs in the album	Albums are longer; singles = single track length
`Fitness_for_Clubs`	Average of Danceability, Energy, Valence + normalized Loudness	Measures how suitable a song is for energetic environments
`Likes_to_Views`	Likes ÷ Views	Indicates audience engagement and song appeal
`Stream_to_Views`	Spotify Streams ÷ YouTube Views	Shows which platform is more dominant for a song
`Comments_to_Likes`	Comments ÷ Likes	Captures how expressive or controversial a song is
`Loudness_High`	Boolean: True if Loudness is above the dataset median	Singles are often louder (commercial mastering)
`Danceability_Valence`	Danceability × Valence	Indicates upbeat/feel-good potential
`Popular_Site`	Categorical: 'YouTube' if Views > Streams, else 'Spotify'	Helps identify platform audience bias
`Is_Remix`	Boolean: True if 'remix' appears in title, track, or description	Remixes may follow different release patterns
`Streams_per_Minute`	Streams ÷ (Duration in minutes)	Highlights songs with replay value or viral potential

	count	mean	std	min	25%	50%	75%	max
Views	19298.0	9.683675e+07	2.791808e+08	26.000000	2.066310e+06	1.558484e+07	7.340811e+07	8.079649e+09
Stream	19298.0	1.381404e+08	2.474362e+08	6574.000000	1.784301e+07	5.026902e+07	1.407806e+08	3.386520e+09
Avg_Artist_Song_Views	19298.0	9.683675e+07	1.594558e+08	3802.800000	1.518029e+07	4.292052e+07	1.075103e+08	1.546021e+09
Streams_per_Minute	19298.0	3.961227e+07	7.332718e+07	1720.582077	4.920396e+06	1.404450e+07	3.981542e+07	1.015753e+09
Likes	19298.0	6.799624e+05	1.815996e+06	0.000000	2.395475e+04	1.317370e+05	5.394230e+05	5.078865e+07
Total_Album_Length	19298.0	6.586782e+05	1.117053e+06	30985.000000	2.318890e+05	4.396870e+05	7.892400e+05	4.123335e+07
Stream_to_Views	19298.0	2.601677e+03	2.786099e+05	0.000074	1.113401e+00	3.066113e+00	1.218862e+01	3.863756e+07
Comments	19298.0	2.822475e+04	1.971631e+05	0.000000	5.580000e+02	3.456500e+03	1.478250e+04	1.608314e+07
Duration_ms	19298.0	2.247218e+05	1.275723e+05	30985.000000	1.802432e+05	2.133575e+05	2.519268e+05	4.676058e+06
Tempo	19298.0	1.205809e+02	2.957300e+01	0.000000	9.699750e+01	1.199650e+02	1.399405e+02	2.433720e+02
Channel_freq	19298.0	1.153695e+01	2.793349e+01	1.000000	2.000000e+00	7.000000e+00	1.000000e+01	2.380000e+02
Loudness	19298.0	-7.622436e+00	4.618275e+00	-46.251000	-8.756000e+00	-6.506000e+00	-4.922000e+00	9.200000e-01
Key	19298.0	5.292103e+00	3.579583e+00	0.000000	2.000000e+00	5.000000e+00	8.000000e+00	1.100000e+01
Album_Song_Count	19298.0	2.894808e+00	3.011082e+00	1.000000	1.000000e+00	2.000000e+00	3.000000e+00	2.800000e+01
Log_Views	19298.0	1.614225e+01	2.723626e+00	3.295837	1.454128e+01	1.656181e+01	1.811155e+01	2.281261e+01
Log_Comments	19298.0	7.757986e+00	2.722050e+00	0.000000	6.326149e+00	8.148301e+00	9.601267e+00	1.659328e+01
Song_Name_Length	19298.0	3.666805e+00	2.681341e+00	1.000000	2.000000e+00	3.000000e+00	5.000000e+00	4.100000e+01
Log_Likes	19298.0	1.143761e+01	2.555692e+00	0.000000	1.008396e+01	1.178857e+01	1.319826e+01	1.774318e+01
Log_Stream_to_Views	19298.0	1.972495e+00	1.796072e+00	0.000074	7.482986e-01	1.402688e+00	2.579354e+00	1.746974e+01
Log_Streams_per_Minute	19298.0	1.638836e+01	1.646517e+00	7.450999	1.540890e+01	1.645774e+01	1.749976e+01	2.073890e+01
Log_Stream	19298.0	1.765379e+01	1.646086e+00	8.791030	1.669712e+01	1.773290e+01	1.876271e+01	2.194307e+01
Log_Avg_Artist_Song_Views	19298.0	1.739971e+01	1.622488e+00	8.243756	1.653551e+01	1.757486e+01	1.849310e+01	2.115895e+01
Artist_freq	19298.0	9.611048e+00	8.479381e-01	1.000000	9.000000e+00	1.000000e+01	1.000000e+01	1.000000e+01
Log_Total_Album_Length	19298.0	1.303195e+01	7.876977e-01	10.341291	1.235402e+01	1.299382e+01	1.357883e+01	1.753476e+01
Log_Album_Song_Count	19298.0	1.194217e+00	5.206706e-01	0.693147	6.931472e-01	1.098612e+00	1.386294e+00	3.367296e+00
Licensed	19298.0	7.128718e-01	4.524336e-01	0.000000	0.000000e+00	1.000000e+00	1.000000e+00	1.000000e+00
Album_type_Label	19298.0	2.411131e-01	4.277698e-01	0.000000	0.000000e+00	0.000000e+00	0.000000e+00	1.000000e+00
Popular_Site	19298.0	2.256192e-01	4.180003e-01	0.000000	0.000000e+00	0.000000e+00	0.000000e+00	1.000000e+00
official_video	19298.0	7.921028e-01	4.058134e-01	0.000000	1.000000e+00	1.000000e+00	1.000000e+00	1.000000e+00
Log_Duration_ms	19298.0	1.226753e+01	3.112167e-01	10.341291	1.210207e+01	1.227073e+01	1.243690e+01	1.535797e+01
Acousticness	19298.0	2.882187e-01	2.859003e-01	0.000001	4.360000e-02	1.880000e-01	4.690000e-01	9.960000e-01
Valence	19298.0	5.283075e-01	2.452507e-01	0.000000	3.380000e-01	5.350000e-01	7.247500e-01	9.930000e-01
Energy	19298.0	6.358078e-01	2.135669e-01	0.000020	5.100000e-01	6.670000e-01	7.970000e-01	1.000000e+00
Danceability_Valence	19298.0	3.469746e-01	2.006129e-01	0.000000	1.836630e-01	3.303950e-01	4.954723e-01	9.321880e-01
Instrumentalness	19298.0	5.565527e-02	1.930548e-01	0.000000	0.000000e+00	2.410000e-06	4.420000e-04	1.000000e+00
Danceability	19298.0	6.210537e-01	1.655111e-01	0.000000	5.200000e-01	6.390000e-01	7.420000e-01	9.750000e-01
Liveness	19298.0	1.912131e-01	1.651456e-01	0.014500	9.402500e-02	1.250000e-01	2.340000e-01	1.000000e+00
Fitness_for_Clubs	19298.0	6.510185e-01	1.360836e-01	0.066360	5.816645e-01	6.717617e-01	7.473616e-01	9.327136e-01
Speechiness	19298.0	9.471736e-02	1.047307e-01	0.000000	3.570000e-02	5.050000e-02	1.037500e-01	9.640000e-01
Comments_to_Likes	19298.0	3.392488e-02	4.035674e-02	0.000000	1.843275e-02	2.792616e-02	4.106224e-02	2.828808e+00
Log_Comments_to_Likes	19298.0	3.283890e-02	3.018017e-02	0.000000	1.826492e-02	2.754334e-02	4.024157e-02	1.342553e+00
Likes_to_Views	19298.0	1.212797e-02	1.116786e-02	0.000000	5.628699e-03	8.699882e-03	1.489781e-02	2.492042e-01
Log_Likes_to_Views	19298.0	1.199608e-02	1.077273e-02	0.000000	5.612917e-03	8.662256e-03	1.478792e-02	2.225067e-01

Algorithm	Silhouette Score
K-Means (K=3)	0.42
Agglomerative (K=3)	0.39

Model	Validation Accuracy	Val Macro F1	Test Accuracy	Test Macro F1
Random Forest	87.3%	0.82	86.7%	0.81
Gradient Boosting	86.5%	0.81	83.2%	0.78–0.79
SVM	86.0%	0.82	83.8%	0.79
Voting Ensemble	87.4%	0.82	87.1%	0.82

Metric	Value
Accuracy	79%
Macro F1 Score	0.79
Precision	0.80 (class 0), 0.78 (class 1)
Recall	0.77 (class 0), 0.80 (class 1)

Machine Learning - Assignment 2¶

Spotify and Youtube Dataset Analysis¶

Part A:Data Exploration & Visualization¶

Data Loading and Initial Exploration¶

Understanding the Dataset¶

Here’s an overview of how all major features interact Likes, Views, Comments, Streams and how they differ between singles and albums.¶

Section B - Data Pre-processing¶

Data preperation:¶

Cleaning Before Feature Engineering:¶

feature engineering¶

Data Cleaning Part 2:¶

Full Data Cleaning and Preprocessing Summary¶

1. Imputation and Filtering¶

2. Feature Engineering¶

3. Log Transformation¶

4. Encoding Categorical Features¶

5. Feature Exclusion¶

6. Final Verification¶

Part C:¶

Section C.1 - Setup and Data Preparation¶

Section C.2 - Model: Random Forest¶

Section C.3 - Model: Gradient Boosting¶

Section C.4 - Model: SVM (with scaling)¶

Section C.5 - VotingClassifier Ensemble¶

Section C.6 - Model Comparison and Evaluation¶

Final Evaluation Summary¶

Key Insights:¶

Section C.7 - Feature Importance (Random Forest)¶

Section C.8 - Drop Low Importance Features and Re-evaluate¶

Section D:¶

Feature Selection and Standardization¶

Feature Standardization¶

Choosing e Optimal Number of Clusters¶

Comparing DBSCAN and Final K-Means Clustering¶

Observation:¶

Visualizing Clusters with PCA¶

Interpreting Cluster Profiles¶

Key Features Driving Cluster Differences¶

Visual Exploration of Clusters¶

Why We Skipped DBSCAN¶

Agglomerative Clustering¶

Clustering Evaluation (Silhouette Scores)¶

Understanding Silhouette Scores¶

Part E – Exploring Artist Engagement¶

Artist-Level Feature Descriptions¶

Modeling Artist Engagement¶

Goal:¶

Modeling Approach:¶

Model Evaluation – Predicting High Engagement Artists¶

Performance Summary:¶

Confusion Matrix Observations:¶

Test Set¶

Train Set¶

Insight:¶

Gradient Boosting Model: Predicting High Engagement Artists¶

What We Are Predicting¶

Final GBoost Model Tuned and Optimized¶

Final Evaluation (Test Set)¶